{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import gc\n",
    "import numpy as np \n",
    "import pandas as pd \n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "from sklearn.linear_model import LogisticRegression, RidgeClassifier, Ridge, HuberRegressor\n",
    "from sklearn.feature_selection import RFE\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stacked = pd.read_csv('../oofs/kain-train-features-v0.1.2.csv', index_col=0)\n",
    "test_stacked = pd.read_csv('../oofs/kain-test-features-v0.1.2.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LGB_cat_0_7956629719199435_csv</th>\n",
       "      <th>LGB_cat_0_796965615822933_csv</th>\n",
       "      <th>LGB_cat_0_7971758465559547_csv</th>\n",
       "      <th>LGB_cat_0_7971922012487902_csv</th>\n",
       "      <th>LGB_cat_0_7975664475835779_csv</th>\n",
       "      <th>LGB_cat_0_7982459487910958_csv</th>\n",
       "      <th>LGB_cat_0_7985379016892897_csv</th>\n",
       "      <th>LGB_cat_0_7986254088147751_csv</th>\n",
       "      <th>LGB_cat_0_7987384223756171_csv</th>\n",
       "      <th>LGB_cat_0_7988757981190066_csv</th>\n",
       "      <th>...</th>\n",
       "      <th>TARGET.72</th>\n",
       "      <th>TARGET.73</th>\n",
       "      <th>TARGET.74</th>\n",
       "      <th>TARGET.75</th>\n",
       "      <th>TARGET.76</th>\n",
       "      <th>TARGET.77</th>\n",
       "      <th>TARGET.78</th>\n",
       "      <th>TARGET.79</th>\n",
       "      <th>TARGET.80</th>\n",
       "      <th>TARGET.81</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.263165</td>\n",
       "      <td>0.210463</td>\n",
       "      <td>0.254966</td>\n",
       "      <td>0.272226</td>\n",
       "      <td>0.242135</td>\n",
       "      <td>0.217145</td>\n",
       "      <td>0.170371</td>\n",
       "      <td>0.194649</td>\n",
       "      <td>0.213564</td>\n",
       "      <td>0.166478</td>\n",
       "      <td>...</td>\n",
       "      <td>0.673457</td>\n",
       "      <td>0.577123</td>\n",
       "      <td>0.615324</td>\n",
       "      <td>0.409131</td>\n",
       "      <td>0.339133</td>\n",
       "      <td>0.310050</td>\n",
       "      <td>0.292603</td>\n",
       "      <td>0.274747</td>\n",
       "      <td>0.229829</td>\n",
       "      <td>0.377313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.028534</td>\n",
       "      <td>0.022802</td>\n",
       "      <td>0.021364</td>\n",
       "      <td>0.027252</td>\n",
       "      <td>0.025308</td>\n",
       "      <td>0.024883</td>\n",
       "      <td>0.021984</td>\n",
       "      <td>0.021119</td>\n",
       "      <td>0.023010</td>\n",
       "      <td>0.023200</td>\n",
       "      <td>...</td>\n",
       "      <td>0.103917</td>\n",
       "      <td>0.039187</td>\n",
       "      <td>0.074826</td>\n",
       "      <td>0.037902</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.023711</td>\n",
       "      <td>0.018794</td>\n",
       "      <td>0.009405</td>\n",
       "      <td>0.019214</td>\n",
       "      <td>0.017564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.044988</td>\n",
       "      <td>0.035597</td>\n",
       "      <td>0.037636</td>\n",
       "      <td>0.041098</td>\n",
       "      <td>0.039781</td>\n",
       "      <td>0.038767</td>\n",
       "      <td>0.047803</td>\n",
       "      <td>0.032391</td>\n",
       "      <td>0.034830</td>\n",
       "      <td>0.045556</td>\n",
       "      <td>...</td>\n",
       "      <td>0.098294</td>\n",
       "      <td>0.068005</td>\n",
       "      <td>0.105474</td>\n",
       "      <td>0.038344</td>\n",
       "      <td>0.055681</td>\n",
       "      <td>0.040151</td>\n",
       "      <td>0.030012</td>\n",
       "      <td>0.048608</td>\n",
       "      <td>0.041607</td>\n",
       "      <td>0.024725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.028133</td>\n",
       "      <td>0.023076</td>\n",
       "      <td>0.025890</td>\n",
       "      <td>0.025638</td>\n",
       "      <td>0.025989</td>\n",
       "      <td>0.026900</td>\n",
       "      <td>0.027623</td>\n",
       "      <td>0.021785</td>\n",
       "      <td>0.027436</td>\n",
       "      <td>0.025201</td>\n",
       "      <td>...</td>\n",
       "      <td>0.035021</td>\n",
       "      <td>0.080346</td>\n",
       "      <td>0.060589</td>\n",
       "      <td>0.016141</td>\n",
       "      <td>0.048706</td>\n",
       "      <td>0.051309</td>\n",
       "      <td>0.037157</td>\n",
       "      <td>0.041925</td>\n",
       "      <td>0.030234</td>\n",
       "      <td>0.035943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.061272</td>\n",
       "      <td>0.049731</td>\n",
       "      <td>0.047512</td>\n",
       "      <td>0.059441</td>\n",
       "      <td>0.059825</td>\n",
       "      <td>0.053356</td>\n",
       "      <td>0.059691</td>\n",
       "      <td>0.043630</td>\n",
       "      <td>0.050947</td>\n",
       "      <td>0.054172</td>\n",
       "      <td>...</td>\n",
       "      <td>0.129976</td>\n",
       "      <td>0.115208</td>\n",
       "      <td>0.147167</td>\n",
       "      <td>0.083344</td>\n",
       "      <td>0.075375</td>\n",
       "      <td>0.100184</td>\n",
       "      <td>0.088149</td>\n",
       "      <td>0.061036</td>\n",
       "      <td>0.056570</td>\n",
       "      <td>0.059047</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 154 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   LGB_cat_0_7956629719199435_csv  LGB_cat_0_796965615822933_csv  \\\n",
       "0                        0.263165                       0.210463   \n",
       "1                        0.028534                       0.022802   \n",
       "2                        0.044988                       0.035597   \n",
       "3                        0.028133                       0.023076   \n",
       "4                        0.061272                       0.049731   \n",
       "\n",
       "   LGB_cat_0_7971758465559547_csv  LGB_cat_0_7971922012487902_csv  \\\n",
       "0                        0.254966                        0.272226   \n",
       "1                        0.021364                        0.027252   \n",
       "2                        0.037636                        0.041098   \n",
       "3                        0.025890                        0.025638   \n",
       "4                        0.047512                        0.059441   \n",
       "\n",
       "   LGB_cat_0_7975664475835779_csv  LGB_cat_0_7982459487910958_csv  \\\n",
       "0                        0.242135                        0.217145   \n",
       "1                        0.025308                        0.024883   \n",
       "2                        0.039781                        0.038767   \n",
       "3                        0.025989                        0.026900   \n",
       "4                        0.059825                        0.053356   \n",
       "\n",
       "   LGB_cat_0_7985379016892897_csv  LGB_cat_0_7986254088147751_csv  \\\n",
       "0                        0.170371                        0.194649   \n",
       "1                        0.021984                        0.021119   \n",
       "2                        0.047803                        0.032391   \n",
       "3                        0.027623                        0.021785   \n",
       "4                        0.059691                        0.043630   \n",
       "\n",
       "   LGB_cat_0_7987384223756171_csv  LGB_cat_0_7988757981190066_csv    ...      \\\n",
       "0                        0.213564                        0.166478    ...       \n",
       "1                        0.023010                        0.023200    ...       \n",
       "2                        0.034830                        0.045556    ...       \n",
       "3                        0.027436                        0.025201    ...       \n",
       "4                        0.050947                        0.054172    ...       \n",
       "\n",
       "   TARGET.72  TARGET.73  TARGET.74  TARGET.75  TARGET.76  TARGET.77  \\\n",
       "0   0.673457   0.577123   0.615324   0.409131   0.339133   0.310050   \n",
       "1   0.103917   0.039187   0.074826   0.037902   0.000000   0.023711   \n",
       "2   0.098294   0.068005   0.105474   0.038344   0.055681   0.040151   \n",
       "3   0.035021   0.080346   0.060589   0.016141   0.048706   0.051309   \n",
       "4   0.129976   0.115208   0.147167   0.083344   0.075375   0.100184   \n",
       "\n",
       "   TARGET.78  TARGET.79  TARGET.80  TARGET.81  \n",
       "0   0.292603   0.274747   0.229829   0.377313  \n",
       "1   0.018794   0.009405   0.019214   0.017564  \n",
       "2   0.030012   0.048608   0.041607   0.024725  \n",
       "3   0.037157   0.041925   0.030234   0.035943  \n",
       "4   0.088149   0.061036   0.056570   0.059047  \n",
       "\n",
       "[5 rows x 154 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_stacked.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LGB_cat_0_7956629719199435_csv</th>\n",
       "      <th>LGB_cat_0_796965615822933_csv</th>\n",
       "      <th>LGB_cat_0_7971758465559547_csv</th>\n",
       "      <th>LGB_cat_0_7971922012487902_csv</th>\n",
       "      <th>LGB_cat_0_7975664475835779_csv</th>\n",
       "      <th>LGB_cat_0_7982459487910958_csv</th>\n",
       "      <th>LGB_cat_0_7985379016892897_csv</th>\n",
       "      <th>LGB_cat_0_7986254088147751_csv</th>\n",
       "      <th>LGB_cat_0_7987384223756171_csv</th>\n",
       "      <th>LGB_cat_0_7988757981190066_csv</th>\n",
       "      <th>...</th>\n",
       "      <th>TARGET.72</th>\n",
       "      <th>TARGET.73</th>\n",
       "      <th>TARGET.74</th>\n",
       "      <th>TARGET.75</th>\n",
       "      <th>TARGET.76</th>\n",
       "      <th>TARGET.77</th>\n",
       "      <th>TARGET.78</th>\n",
       "      <th>TARGET.79</th>\n",
       "      <th>TARGET.80</th>\n",
       "      <th>TARGET.81</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.032046</td>\n",
       "      <td>0.034213</td>\n",
       "      <td>0.036182</td>\n",
       "      <td>0.030748</td>\n",
       "      <td>0.030909</td>\n",
       "      <td>0.030198</td>\n",
       "      <td>0.026424</td>\n",
       "      <td>0.033712</td>\n",
       "      <td>0.032925</td>\n",
       "      <td>0.027859</td>\n",
       "      <td>...</td>\n",
       "      <td>0.060607</td>\n",
       "      <td>0.085077</td>\n",
       "      <td>0.059713</td>\n",
       "      <td>0.015011</td>\n",
       "      <td>0.093184</td>\n",
       "      <td>0.112115</td>\n",
       "      <td>0.059259</td>\n",
       "      <td>0.055032</td>\n",
       "      <td>0.038632</td>\n",
       "      <td>0.055209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.121353</td>\n",
       "      <td>0.118384</td>\n",
       "      <td>0.121478</td>\n",
       "      <td>0.120025</td>\n",
       "      <td>0.119085</td>\n",
       "      <td>0.120870</td>\n",
       "      <td>0.117530</td>\n",
       "      <td>0.117468</td>\n",
       "      <td>0.114038</td>\n",
       "      <td>0.112407</td>\n",
       "      <td>...</td>\n",
       "      <td>0.334951</td>\n",
       "      <td>0.236243</td>\n",
       "      <td>0.293807</td>\n",
       "      <td>0.141847</td>\n",
       "      <td>0.159056</td>\n",
       "      <td>0.220960</td>\n",
       "      <td>0.127498</td>\n",
       "      <td>0.134836</td>\n",
       "      <td>0.125285</td>\n",
       "      <td>0.151449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.022355</td>\n",
       "      <td>0.023191</td>\n",
       "      <td>0.024048</td>\n",
       "      <td>0.023358</td>\n",
       "      <td>0.022430</td>\n",
       "      <td>0.024886</td>\n",
       "      <td>0.025242</td>\n",
       "      <td>0.021554</td>\n",
       "      <td>0.023749</td>\n",
       "      <td>0.025666</td>\n",
       "      <td>...</td>\n",
       "      <td>0.063316</td>\n",
       "      <td>0.042244</td>\n",
       "      <td>0.063244</td>\n",
       "      <td>0.044956</td>\n",
       "      <td>0.052548</td>\n",
       "      <td>0.016997</td>\n",
       "      <td>0.037535</td>\n",
       "      <td>0.042209</td>\n",
       "      <td>0.022144</td>\n",
       "      <td>0.022888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.022851</td>\n",
       "      <td>0.026334</td>\n",
       "      <td>0.027283</td>\n",
       "      <td>0.026948</td>\n",
       "      <td>0.026514</td>\n",
       "      <td>0.028141</td>\n",
       "      <td>0.023710</td>\n",
       "      <td>0.027926</td>\n",
       "      <td>0.029214</td>\n",
       "      <td>0.020858</td>\n",
       "      <td>...</td>\n",
       "      <td>0.130619</td>\n",
       "      <td>0.080953</td>\n",
       "      <td>0.120677</td>\n",
       "      <td>0.047268</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.043664</td>\n",
       "      <td>0.035744</td>\n",
       "      <td>0.067668</td>\n",
       "      <td>0.034352</td>\n",
       "      <td>0.034921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.145823</td>\n",
       "      <td>0.131313</td>\n",
       "      <td>0.126619</td>\n",
       "      <td>0.139883</td>\n",
       "      <td>0.137116</td>\n",
       "      <td>0.142311</td>\n",
       "      <td>0.115298</td>\n",
       "      <td>0.122861</td>\n",
       "      <td>0.141986</td>\n",
       "      <td>0.114807</td>\n",
       "      <td>...</td>\n",
       "      <td>0.304686</td>\n",
       "      <td>0.332027</td>\n",
       "      <td>0.250013</td>\n",
       "      <td>0.103591</td>\n",
       "      <td>0.111472</td>\n",
       "      <td>0.136448</td>\n",
       "      <td>0.118943</td>\n",
       "      <td>0.143444</td>\n",
       "      <td>0.141113</td>\n",
       "      <td>0.118314</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 154 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   LGB_cat_0_7956629719199435_csv  LGB_cat_0_796965615822933_csv  \\\n",
       "0                        0.032046                       0.034213   \n",
       "1                        0.121353                       0.118384   \n",
       "2                        0.022355                       0.023191   \n",
       "3                        0.022851                       0.026334   \n",
       "4                        0.145823                       0.131313   \n",
       "\n",
       "   LGB_cat_0_7971758465559547_csv  LGB_cat_0_7971922012487902_csv  \\\n",
       "0                        0.036182                        0.030748   \n",
       "1                        0.121478                        0.120025   \n",
       "2                        0.024048                        0.023358   \n",
       "3                        0.027283                        0.026948   \n",
       "4                        0.126619                        0.139883   \n",
       "\n",
       "   LGB_cat_0_7975664475835779_csv  LGB_cat_0_7982459487910958_csv  \\\n",
       "0                        0.030909                        0.030198   \n",
       "1                        0.119085                        0.120870   \n",
       "2                        0.022430                        0.024886   \n",
       "3                        0.026514                        0.028141   \n",
       "4                        0.137116                        0.142311   \n",
       "\n",
       "   LGB_cat_0_7985379016892897_csv  LGB_cat_0_7986254088147751_csv  \\\n",
       "0                        0.026424                        0.033712   \n",
       "1                        0.117530                        0.117468   \n",
       "2                        0.025242                        0.021554   \n",
       "3                        0.023710                        0.027926   \n",
       "4                        0.115298                        0.122861   \n",
       "\n",
       "   LGB_cat_0_7987384223756171_csv  LGB_cat_0_7988757981190066_csv    ...      \\\n",
       "0                        0.032925                        0.027859    ...       \n",
       "1                        0.114038                        0.112407    ...       \n",
       "2                        0.023749                        0.025666    ...       \n",
       "3                        0.029214                        0.020858    ...       \n",
       "4                        0.141986                        0.114807    ...       \n",
       "\n",
       "   TARGET.72  TARGET.73  TARGET.74  TARGET.75  TARGET.76  TARGET.77  \\\n",
       "0   0.060607   0.085077   0.059713   0.015011   0.093184   0.112115   \n",
       "1   0.334951   0.236243   0.293807   0.141847   0.159056   0.220960   \n",
       "2   0.063316   0.042244   0.063244   0.044956   0.052548   0.016997   \n",
       "3   0.130619   0.080953   0.120677   0.047268   0.000000   0.043664   \n",
       "4   0.304686   0.332027   0.250013   0.103591   0.111472   0.136448   \n",
       "\n",
       "   TARGET.78  TARGET.79  TARGET.80  TARGET.81  \n",
       "0   0.059259   0.055032   0.038632   0.055209  \n",
       "1   0.127498   0.134836   0.125285   0.151449  \n",
       "2   0.037535   0.042209   0.022144   0.022888  \n",
       "3   0.035744   0.067668   0.034352   0.034921  \n",
       "4   0.118943   0.143444   0.141113   0.118314  \n",
       "\n",
       "[5 rows x 154 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_stacked.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('../../data/application_train.csv')\n",
    "y = train['TARGET']\n",
    "\n",
    "n_train = train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features = train_stacked.values\n",
    "test_features = test_stacked.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LogisticRegression(C=100, class_weight={0:1, 1:3},\n",
    "                               fit_intercept=False, n_jobs=1, max_iter=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "selector = RFE(model, 100, step=2)\n",
    "selector = selector.fit(train_features, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_features = [i for i, y in enumerate(selector.ranking_) if y == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0,\n",
       " 1,\n",
       " 2,\n",
       " 3,\n",
       " 7,\n",
       " 9,\n",
       " 12,\n",
       " 13,\n",
       " 17,\n",
       " 18,\n",
       " 19,\n",
       " 20,\n",
       " 22,\n",
       " 23,\n",
       " 26,\n",
       " 28,\n",
       " 29,\n",
       " 31,\n",
       " 37,\n",
       " 38,\n",
       " 39,\n",
       " 41,\n",
       " 42,\n",
       " 43,\n",
       " 44,\n",
       " 45,\n",
       " 46,\n",
       " 47,\n",
       " 48,\n",
       " 49,\n",
       " 50,\n",
       " 52,\n",
       " 53,\n",
       " 54,\n",
       " 55,\n",
       " 56,\n",
       " 57,\n",
       " 59,\n",
       " 61,\n",
       " 62,\n",
       " 63,\n",
       " 64,\n",
       " 65,\n",
       " 66,\n",
       " 67,\n",
       " 68,\n",
       " 69,\n",
       " 70,\n",
       " 72,\n",
       " 74,\n",
       " 76,\n",
       " 78,\n",
       " 80,\n",
       " 82,\n",
       " 83,\n",
       " 84,\n",
       " 85,\n",
       " 86,\n",
       " 87,\n",
       " 88,\n",
       " 91,\n",
       " 92,\n",
       " 93,\n",
       " 94,\n",
       " 95,\n",
       " 96,\n",
       " 97,\n",
       " 102,\n",
       " 104,\n",
       " 105,\n",
       " 106,\n",
       " 108,\n",
       " 109,\n",
       " 110,\n",
       " 112,\n",
       " 114,\n",
       " 115,\n",
       " 116,\n",
       " 117,\n",
       " 118,\n",
       " 119,\n",
       " 122,\n",
       " 126,\n",
       " 129,\n",
       " 131,\n",
       " 132,\n",
       " 133,\n",
       " 134,\n",
       " 135,\n",
       " 138,\n",
       " 140,\n",
       " 142,\n",
       " 143,\n",
       " 146,\n",
       " 147,\n",
       " 148,\n",
       " 149,\n",
       " 150,\n",
       " 151,\n",
       " 152]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(307511, 154)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Modeling Stage\n",
      "TRAIN:  [     0      1      2 ... 307505 307506 307508] TEST:  [     9     16     25 ... 307507 307509 307510]\n",
      "1   Fold AUC : 0.8031452843225432\n",
      "2   Fold AUC : 0.8032109323723113\n",
      "3   Fold AUC : 0.8032345241916832\n",
      "4   Fold AUC : 0.8032450373522547\n",
      "5   Fold AUC : 0.8032521669256527\n",
      "Fold AUC : 0.8032247412113762\n",
      "TRAIN:  [     0      2      3 ... 307508 307509 307510] TEST:  [     1      8     17 ... 307497 307503 307504]\n",
      "1   Fold AUC : 0.8073915089584248\n",
      "2   Fold AUC : 0.807404409629424\n",
      "3   Fold AUC : 0.8073974514785667\n",
      "4   Fold AUC : 0.8073868197698038\n",
      "5   Fold AUC : 0.8073813202375268\n",
      "Fold AUC : 0.8074020002076208\n",
      "TRAIN:  [     0      1      4 ... 307507 307509 307510] TEST:  [     2      3      6 ... 307494 307501 307508]\n",
      "1   Fold AUC : 0.8062123778133085\n",
      "2   Fold AUC : 0.8061983310985235\n",
      "3   Fold AUC : 0.8061836823307842\n",
      "4   Fold AUC : 0.8061687663797813\n",
      "5   Fold AUC : 0.8061537257432553\n",
      "Fold AUC : 0.8061915767056165\n",
      "TRAIN:  [     0      1      2 ... 307508 307509 307510] TEST:  [     7     11     13 ... 307495 307498 307506]\n",
      "1   Fold AUC : 0.8005673894662465\n",
      "2   Fold AUC : 0.8006908445741454\n",
      "3   Fold AUC : 0.8007500232988447\n",
      "4   Fold AUC : 0.800777049492539\n",
      "5   Fold AUC : 0.8007958735465368\n",
      "Fold AUC : 0.8007223805107324\n",
      "TRAIN:  [     1      2      3 ... 307508 307509 307510] TEST:  [     0      4      5 ... 307499 307502 307505]\n",
      "1   Fold AUC : 0.806166830990529\n",
      "2   Fold AUC : 0.8060736228295958\n",
      "3   Fold AUC : 0.8060178134127891\n",
      "4   Fold AUC : 0.8059816494672271\n",
      "5   Fold AUC : 0.8059554508678259\n",
      "Fold AUC : 0.8060481772834038\n",
      "AVERAGED AUC : 0.8047177751837499\n"
     ]
    }
   ],
   "source": [
    "aucs = []\n",
    "test_set = []\n",
    "validation_set = []\n",
    "print(\"\\nModeling Stage\")\n",
    "\n",
    "kf = KFold(n_splits=5, random_state=1002, shuffle=True)\n",
    "kf.get_n_splits(train_features)\n",
    "\n",
    "n_bagged = 6\n",
    "\n",
    "for train_index, test_index in kf.split(train_features):\n",
    "    print(\"TRAIN: \", train_index, \"TEST: \", test_index)\n",
    "\n",
    "    X = train_features\n",
    "    y_ = y.values\n",
    "    x_train, x_test = X[train_index][:, selected_features], X[test_index][:, selected_features]\n",
    "    y_train, y_test = y_[train_index], y_[test_index]\n",
    "    \n",
    "    oof_baggs = np.zeros([n_bagged , x_test.shape[0]])\n",
    "    preds_baggs = np.zeros([n_bagged , test_features.shape[0]])\n",
    "    \n",
    "    for _it in range(1, n_bagged):\n",
    "        \n",
    "        model = LogisticRegression(C=0.1 * _it, class_weight={0: 1, 1: 5},\n",
    "                               fit_intercept=False, n_jobs=1, max_iter=10000)\n",
    "        model.fit(x_train, y_train)\n",
    "        \n",
    "        print(_it,' ' ,'Fold AUC :', roc_auc_score(y_test, model.predict_proba(x_test)[:, 1]))\n",
    "        oof_baggs[_it, :] = model.predict_proba(x_test)[:, 1]\n",
    "        preds_baggs[_it, :] = model.predict_proba(test_features[:, selected_features])[:, 1]\n",
    "        \n",
    "    val_preds = pd.DataFrame(oof_baggs).T\n",
    "    test_preds = pd.DataFrame(preds_baggs).T\n",
    "    \n",
    "    print('Fold AUC :', roc_auc_score(y_test, val_preds.rank(axis=0, method='min').mul(val_preds.shape[1] * [1 / val_preds.shape[1]]).sum(1) / val_preds.shape[0]))\n",
    "    aucs.append(roc_auc_score(y_test, val_preds.rank(axis=0, method='min').mul(val_preds.shape[1] * [1 / val_preds.shape[1]]).sum(1) / val_preds.shape[0]))\n",
    "   \n",
    "    test_set.append(test_preds.rank(axis=0, method='min').mul(test_preds.shape[1] * [1 / test_preds.shape[1]]).sum(1) / test_preds.shape[0])\n",
    "    gc.collect()\n",
    "        \n",
    "print('AVERAGED AUC :', np.mean(aucs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds = pd.DataFrame(test_set).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100001</td>\n",
       "      <td>0.363733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100005</td>\n",
       "      <td>0.806692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100013</td>\n",
       "      <td>0.394059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100028</td>\n",
       "      <td>0.380761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100038</td>\n",
       "      <td>0.785200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR    TARGET\n",
       "0      100001  0.363733\n",
       "1      100005  0.806692\n",
       "2      100013  0.394059\n",
       "3      100028  0.380761\n",
       "4      100038  0.785200"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_hat = test_preds.rank(axis=0, method='min').mul(test_preds.shape[1] * [1 / test_preds.shape[1]]).sum(1) / test_preds.shape[0] \n",
    "    \n",
    "\n",
    "\n",
    "y_hat.head(10)\n",
    "\n",
    "\n",
    "sampl_sub = pd.read_csv('../../data/sample_submission.csv')\n",
    "\n",
    "\n",
    "sampl_sub['TARGET'] = y_hat.values\n",
    "\n",
    "\n",
    "\n",
    "sampl_sub.to_csv(\"ridge-stack-submission.csv\", index=False)\n",
    "\n",
    "\n",
    "\n",
    "sampl_sub.head()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
